package com.spider.silence.worker;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.spider.silence.abstarct.downloader.AbstractCommonDownLoader;
import com.spider.silence.proxy.ProxyPool;
import com.spider.silence.task.SinaNewsModuleTask;
import com.spider.silence.task.SinaNewsTask;
import com.spider.silence.task.manager.CommonTaskManager;
import com.spider.silence.utils.PropertiesUtil;
import org.apache.log4j.Logger;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;

import java.net.URLDecoder;
import java.util.Date;

/**
 * @author Administrator
 * @description 描述
 * @time 2017/4/12 0012
 */
public class SinaNewsModuleWorker  extends AbstractCommonDownLoader implements Runnable{
    private Logger logger = Logger.getLogger(SinaNewsModuleWorker.class);

    @Override
    public void run() {
        while (true && !Thread.interrupted()) {
            try {
                SinaNewsModuleTask moduleTask = (SinaNewsModuleTask) CommonTaskManager.takeTask("SinaNewsModule", 1000);
                if(null!=moduleTask) {
                    String url = moduleTask.getModuleUrl();
                    String name = moduleTask.getChannel();
                    int pageSize = 22;
                    if ("国内".equals(name)) {
                        String news_type = "gnxw";
                        String[] types = {"gdxw1", "gatxw", "zs-pl", "mtjj"};
                    /*Date now1 = new Date();
                    Calendar cal = Calendar.getInstance();
                    cal.setTime(now1);
                    cal.add(Calendar.HOUR_OF_DAY,-1);*/
                        Date now = new Date();
                        // int pageSize = 22;
                        for (int m = 0; m < types.length; m++) {
                            boolean flag = true;
                            long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));

                            while (flag && remianing >= 0) {
                                try {
                                    remianing = future - System.currentTimeMillis();
                                    String url1 = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=" + news_type + "&cat_2=" + types[m] + "&level==1%7c%7c=2&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1&format=json&page=1&callback=newsloadercallback&_=" + now.getTime();
                                    Document document = get(url1, ProxyPool.proxy());
                                    //System.out.println(document.toString());
                                    Elements body = document.select("body");
                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    JSONObject object = JSON.parseObject(body.toString().substring(begin + 1, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//
//                                            try {
//                                                now = new Date();
//                                                String url2 = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=" + news_type + "&cat_2=" + types[m] + "&level==1%7c%7c=2&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1&format=json&page=" + j + "&callback=newsloadercallback&_=" + now.getTime();
//                                                Document news_document = get(url2, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                JSONObject object1 = JSON.parseObject(body1.toString().substring(begin1 + 1, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//                                        }
//                                    }
                                    flag = false;

                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }
                        }
                    } else if ("国际".equals(name)) {

                        String news_type = "gjxw";
                        String[] types = {"gj-yz", "gj-oz", "gj-mz"};

                        Date now = new Date();

                        String url1 = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=" + news_type + "&level==1%7c%7c=2&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1&format=json&page=1&callback=newsloadercallback&_=" + now.getTime();//最新新闻
                        boolean flag1 = true;
                        long future1 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                        long remianing1 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                        while (flag1 && remianing1 >= 0) {
                            try {
                                remianing1 = future1 - System.currentTimeMillis();
                                Document document = get(url1, ProxyPool.proxy());
                                //System.out.println(document.toString());
                                Elements body = document.select("body");
                                int begin = body.toString().indexOf("(");
                                int end = body.toString().lastIndexOf(")");

                                JSONObject object = JSON.parseObject(body.toString().substring(begin + 1, end));
                                JSONObject resultObject = object.getJSONObject("result");
                                String total = resultObject.get("total").toString();
                                JSONArray urlArray = resultObject.getJSONArray("data");

                                for (int n = 0; n < urlArray.size(); n++) {
                                    JSONObject contentObject = urlArray.getJSONObject(n);
                                    String news_url = contentObject.get("url").toString();
                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                        CommonTaskManager.addTask("SinaNews", task, 1000);
                                        System.out.println(news_url);
                                    }
                                }

//                                int total_count = 0;
//                                int total_page = 0;
//                                try {
//                                    total_count = Integer.parseInt(total);
//                                    total_page = (total_count - 1) / pageSize + 1;
//                                } catch (Exception e) {
//                                    e.printStackTrace();
//                                }
//
//                                if (total_page >= 2) {
//                                    for (int j = 2; j <= total_page; j++) {
//                                        try {
//                                            now = new Date();
//                                            String new_url = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=" + news_type + "&level==1%7c%7c=2&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1&format=json&page=" + j + "&callback=newsloadercallback&_=" + now.getTime();
//                                            Document news_document = get(new_url, ProxyPool.proxy());
//                                            Elements body1 = news_document.select("body");
//                                            int begin1 = body1.toString().indexOf("(");
//                                            int end1 = body1.toString().lastIndexOf(")");
//
//                                            JSONObject object1 = JSON.parseObject(body1.toString().substring(begin1 + 1, end1));
//                                            JSONObject resultObject1 = object1.getJSONObject("result");
//                                            JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                            for (int n = 0; n < urlArray1.size(); n++) {
//                                                JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                String news_url = contentObject.get("url").toString();
//                                                if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                    SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                    CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                    System.out.println(news_url);
//                                                }
//                                            }
//                                        } catch (Exception e) {
//                                            logger.error(e.getMessage());
//                                            e.printStackTrace();
//                                            continue;
//                                        }
//
//                                    }
//                                }

                                flag1 = false;
                            } catch (Exception e) {
                                logger.error(e.getMessage());
                                e.printStackTrace();
                            }


                        }

                        now = new Date();
                        String url2 = "http://api.roll.news.sina.com.cn/zt_list?\n" +
                                "channel=news&cat_1=" + news_type + "&cat_2==hqqw%7c%7c=gjmtjj&level==1%7c%7c\n" +
                                "=2&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1&format=json&page=1&callback=newsloadercallback&_\n" +
                                "=" + now.getTime();
                        boolean flag2 = true;
                        long future2 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                        long remianing2 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));


                        while (flag2 && remianing2 >= 0) {
                            try {
                                remianing2 = future2 - System.currentTimeMillis();
                                Document document = get(url2, ProxyPool.proxy());
                                //System.out.println(document.toString());
                                Elements body = document.select("body");
                                int begin = body.toString().indexOf("(");
                                int end = body.toString().lastIndexOf(")");

                                JSONObject object = JSON.parseObject(body.toString().substring(begin + 1, end));
                                JSONObject resultObject = object.getJSONObject("result");
                                String total = resultObject.get("total").toString();
                                JSONArray urlArray = resultObject.getJSONArray("data");

                                for (int n = 0; n < urlArray.size(); n++) {
                                    JSONObject contentObject = urlArray.getJSONObject(n);
                                    String news_url = contentObject.get("url").toString();
                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                        CommonTaskManager.addTask("SinaNews", task, 1000);
                                        System.out.println(news_url);
                                    }
                                }

//                                int total_count = 0;
//                                int total_page = 0;
//                                try {
//                                    total_count = Integer.parseInt(total);
//                                    total_page = (total_count - 1) / pageSize + 1;
//                                } catch (Exception e) {
//                                    e.printStackTrace();
//                                }
//
//                                if (total_page >= 2) {
//                                    for (int j = 2; j <= total_page; j++) {
//                                        try {
//                                            now = new Date();
//                                            String new_url = "http://api.roll.news.sina.com.cn/zt_list? " +
//                                                    "channel=news&cat_1=" + news_type + "&cat_2==hqqw%7c%7c=gjmtjj&level==1%7c%7c\n" +
//                                                    "=2&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1&format=json&page=" + j + "&callback=newsloadercallback&_\n" +
//                                                    "=" + now.getTime();
//                                            Document news_document = get(new_url, ProxyPool.proxy());
//                                            Elements body1 = news_document.select("body");
//                                            int begin1 = body1.toString().indexOf("(");
//                                            int end1 = body1.toString().lastIndexOf(")");
//
//                                            JSONObject object1 = JSON.parseObject(body1.toString().substring(begin1 + 1, end1));
//                                            JSONObject resultObject1 = object1.getJSONObject("result");
//                                            JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                            for (int n = 0; n < urlArray1.size(); n++) {
//                                                JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                String news_url = contentObject.get("url").toString();
//                                                if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                    SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                    CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                    System.out.println(news_url);
//                                                }
//                                            }
//                                        } catch (Exception e) {
//                                            logger.error(e.getMessage());
//                                            e.printStackTrace();
//                                            continue;
//                                        }
//
//                                    }
//                                }


                                flag2 = false;
                            } catch (Exception e) {
                                logger.error(e.getMessage());
                                e.printStackTrace();
                            }
                        }

                        for (int m = 0; m < types.length; m++) {
                            now = new Date();
                            String url3 = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gjxw&cat_3=" + types[m] +
                                    "&level==1%7c%7c=2&show_ext=1&show_all=1&show_num=" + pageSize +
                                    "&tag=1&format=json&page=1&callback=newsloadercallback&_=" + now.getTime();
                            boolean flag3 = true;
                            long future3 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing3 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));

                            while (flag3 && remianing3 >= 0) {
                                try {
                                    remianing3 = future3 - System.currentTimeMillis();
                                    Document document = get(url3, ProxyPool.proxy());
                                    //System.out.println(document.toString());
                                    Elements body = document.select("body");
                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    JSONObject object = JSON.parseObject(body.toString().substring(begin + 1, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//                                            try {
//                                                now = new Date();
//                                                String new_url = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=gjxw&cat_3=" + types[m] +
//                                                        "&level==1%7c%7c=2&show_ext=1&show_all=1&show_num=" + pageSize +
//                                                        "&tag=1&format=json&page=" + j + "&callback=newsloadercallback&_=" + now.getTime();
//                                                Document news_document = get(new_url, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                JSONObject object1 = JSON.parseObject(body1.toString().substring(begin1 + 1, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//                                        }
//
//
//                                    }
                                    flag3 = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }
                        }

                    } else if ("社会".equals(name)) {
                        String news_type = "shxw";
                        String[] types = {"zqsk%7c%7c=qwys%7c%7c=shwx%7c%7c=fz-shyf", "zqsk", "fz-shyf", "qwys", "shwx"};
                        // int pageSize = 22;
                        Date now = null;

                        for (int m = 0; m < types.length; m++) {
                            now = new Date();

                            String url1 = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=shxw&cat_2==" + types[m] +
                                    "&level==1%7c%7c=2" +
                                    "&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1" +
                                    "&format=json&page=1&callback=newsloadercallback&_=" + now.getTime();
                            boolean flag = true;
                            long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));

                            while (flag && remianing >= 0) {
                                try {
                                    remianing = future - System.currentTimeMillis();
                                    Document document = get(url1, ProxyPool.proxy());
                                    Elements body = document.select("body");
                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    JSONObject object = JSON.parseObject(body.toString().substring(begin + 1, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//                                            try {
//                                                now = new Date();
//                                                String new_url = "http://api.roll.news.sina.com.cn/zt_list?channel=news&cat_1=shxw&cat_2==" + types[m] +
//                                                        "&level==1%7c%7c=2" +
//                                                        "&show_ext=1&show_all=1&show_num=" + pageSize + "&tag=1" +
//                                                        "&format=json&page=" + j + "&callback=newsloadercallback&_=" + now.getTime();
//                                                Document news_document = get(new_url, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                JSONObject object1 = JSON.parseObject(body1.toString().substring(begin1 + 1, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//                                        }
//                                    }

                                    flag = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }
                        }
                    } else if ("军事".equals(name)) {
                        String[] types = {"zgjq", "gjjq"};
                        for (int m = 0; m < types.length; m++) {
                            String url1 = "http://roll.mil.news.sina.com.cn/col/" + types[m] + "/index.shtml";
                            boolean flag = true;
                            long future = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));

                            while (flag && remianing >= 0) {
                                try {
                                    remianing = future - System.currentTimeMillis();
                                    Document document = get(url1, ProxyPool.proxy());
                                    Elements ulList = document.select(".wrap").select("main").select("fixList").select("ul");
                                    for (int n = 0; n < ulList.size(); n++) {
                                        Elements liList = ulList.get(n).select("li");
                                        for (int x = 0; x < liList.size(); x++) {
                                            String news_url = liList.get(x).select("a").get(0).attr("href");
                                            if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                                SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                                CommonTaskManager.addTask("SinaNews", task, 1000);
                                                System.out.println(news_url);
                                            }
                                        }
                                    }

                                    flag = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }

                            }

//                            for (int j = 2; j <= 1000; j++) {
//                                try {
//                                    String new_url = "http://roll.mil.news.sina.com.cn/col/" + types[m] + "/index_" + j + ".shtml";
//                                    Document document = get(new_url, ProxyPool.proxy());
//                                    Elements ulList = document.select(".wrap").select("main").select("fixList").select("ul");
//                                    for (int n = 0; n < ulList.size(); n++) {
//                                        Elements liList = ulList.get(n).select("li");
//                                        for (int x = 0; x < liList.size(); x++) {
//                                            String news_url = liList.get(x).select("a").get(0).attr("href");
//                                            if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                System.out.println(news_url);
//                                            }
//                                        }
//                                    }
//                                } catch (Exception e) {
//                                    logger.error(e.getMessage());
//                                    e.printStackTrace();
//                                    continue;
//                                }
//
//                            }
                        }

                    } else if ("体育".equals(name)) {
                        String url1 = "http://roll.sports.sina.com.cn/s/channel.php?ch=02#col=64&spec=&type=&ch=02&k=&offset_page=0&offset_num=0&num=60&asc=&page=1";

                        try {
                            Document news_document = get(url1, ProxyPool.proxy());
                            Elements ulList = news_document.select("#vBody").select(".wrap").select("#pL_Main").select("#d_list").select("ul");
                            for (int m = 0; m < ulList.size(); m++) {
                                Elements liList = ulList.get(m).select("li");
                                for (int n = 0; n < liList.size(); n++) {
                                    String news_url = liList.get(n).select(".c_tit").select("a").get(0).attr("href");
                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                        CommonTaskManager.addTask("SinaNews", task, 1000);
                                        System.out.println(news_url);
                                    }
                                }
                            }

//                            int total_page = 0;
//                            Elements pageList = news_document.select("#vBody").select(".wrap").select("#pL_Main").select("#d_list").select(".pagebox").select(".pagebox_num");
//                            if (pageList.size() > 0 && pageList != null) {
//                                try {
//                                    total_page = Integer.parseInt(pageList.get(pageList.size() - 1).text());
//                                } catch (Exception e) {
//                                    e.printStackTrace();
//                                }
//                            }
//
//                            if (total_page >= 2) {
//                                for (int j = 2; j <= total_page; j++) {
//                                    try {
//                                        String new_url = "http://roll.sports.sina.com.cn/s/channel.php?ch=02#col=64&spec=&type=&ch=02&k=&offset_page=0&offset_num=0&num=60&asc=&page=" + j;
//                                        Document news_document1 = get(new_url, ProxyPool.proxy());
//                                        Elements ulList1 = news_document1.select("#vBody").select(".wrap").select("#pL_Main").select("#d_list").select("ul");
//                                        for (int m = 0; m < ulList1.size(); m++) {
//                                            Elements liList = ulList1.get(m).select("li");
//                                            for (int n = 0; n < liList.size(); n++) {
//                                                String news_url = liList.get(n).select(".c_tit").select("a").get(0).attr("href");
//                                                if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                    SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                    CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                    System.out.println(news_url);
//                                                }
//
//                                            }
//                                        }
//                                    } catch (Exception e) {
//                                        logger.error(e.getMessage());
//                                        e.printStackTrace();
//                                        continue;
//                                    }
//                                }
//                            }

                        } catch (Exception e) {

                        }


                    } else if ("财经".equals(name)) {
                        int[] lids = {1686, 1687, 1688, 1689, 1690};//国内财经
                        //int pageSize = 20;
                        Date now = null;

                        int[] lids2 = {1693, 1694, 1695, 1696, 1697};//公司财经

                        int[] lids3 = {1703, 1704, 1705, 1706, 1707, 1708};//国内曝光台


                        for (int m = 0; m < lids.length; m++) {
                            now = new Date();
                            String url1 = "http://feed.mix.sina.com.cn/api/roll/get?pageid=155&lid=" + lids[m] + "&num=" + pageSize + "&page=1&callback=feedCardJsonpCallback&_=" + now.getTime();
                            boolean flag1 = true;
                            long future1 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing1 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));


                            while (flag1 && remianing1 >= 0) {
                                try {
                                    remianing1 = future1 - System.currentTimeMillis();
                                    Document document = get(url1, ProxyPool.proxy());
                                    //System.out.println(document.toString());
                                    Elements body = document.select("body");

                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    String new_body = body.toString().substring(begin+1,end);

                                    end = new_body.lastIndexOf(")");


                                   // int end = body.toString().lastIndexOf(")", -8);

                                    JSONObject object = JSON.parseObject(new_body.substring(0, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//                                            try {
//                                                now = new Date();
//                                                String new_url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=155&lid=" + lids[m] + "&num=" + pageSize + "&page=" + j + "&callback=feedCardJsonpCallback&_=" + now.getTime();
//                                                Document news_document = get(new_url, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                String new_body1 = body1.toString().substring(begin1+1,end1);
//
//                                                end1 = new_body1.lastIndexOf(")");
//
//                                                JSONObject object1 = JSON.parseObject(new_body1.toString().substring(0, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//
//                                        }
//                                    }


                                    flag1 = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }

                        }

                        for (int m = 0; m < lids2.length; m++) {
                            now = new Date();
                            String url1 = "http://feed.mix.sina.com.cn/api/roll/get?pageid=164&lid=" + lids2[m] + "&num=" + pageSize + "&page=1&callback=feedCardJsonpCallback&_=" + now.getTime();
                            boolean flag1 = true;
                            long future1 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing1 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));


                            while (flag1 && remianing1 >= 0) {
                                try {
                                    remianing1 = future1 - System.currentTimeMillis();
                                    Document document = get(url1, ProxyPool.proxy());
                                    //System.out.println(document.toString());
                                    Elements body = document.select("body");

                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    String new_body = body.toString().substring(begin+1,end);

                                    end = new_body.lastIndexOf(")");
                                    JSONObject object = JSON.parseObject(new_body.substring(0, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//                                            try {
//                                                now = new Date();
//                                                String new_url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=164&lid=" + lids2[m] + "&num=" + pageSize + "&page=" + j + "&callback=feedCardJsonpCallback&_=" + now.getTime();
//                                                Document news_document = get(new_url, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                String new_body1 = body1.toString().substring(begin1+1,end1);
//                                                end1 = new_body1.lastIndexOf(")");
//
//                                                JSONObject object1 = JSON.parseObject(new_body1.substring(0, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//
//                                        }
//                                    }


                                    flag1 = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }
                        }

                        for (int m = 0; m < lids3.length; m++) {
                            now = new Date();
                            String url1 = "http://feed.mix.sina.com.cn/api/roll/get?pageid=166&lid=" + lids3[m] + "&num=" + pageSize + "&page=1&callback=feedCardJsonpCallback&_=" + now.getTime();
                            boolean flag1 = true;
                            long future1 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing1 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));


                            while (flag1 && remianing1 >= 0) {
                                try {
                                    remianing1 = future1 - System.currentTimeMillis();
                                    Document document = get(url1, ProxyPool.proxy());
                                    //System.out.println(document.toString());
                                    Elements body = document.select("body");

                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    String new_body = body.toString().substring(begin+1,end);

                                    end = new_body.lastIndexOf(")");
                                    JSONObject object = JSON.parseObject(new_body.substring(0, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//                                            try {
//                                                now = new Date();
//                                                String new_url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=166&lid=" + lids3[m] + "&num=" + pageSize + "&page=" + j + "&callback=feedCardJsonpCallback&_=" + now.getTime();
//                                                Document news_document = get(new_url, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                String new_body1 = body1.toString().substring(begin1+1,end1);
//
//                                                end1 = new_body1.lastIndexOf(")");
//                                                JSONObject object1 = JSON.parseObject(new_body1.substring(0, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//
//                                        }
//                                    }


                                    flag1 = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }
                        }


               /* String[] types = {"mzjj", "ozjj", "yzjj", "qtdq"};//国际经济
                for (int m = 0; m < types.length; m++) {
                    try {
                        String url2 = "http://roll.finance.sina.com.cn/finance/gjcj/" + types[m] + "/index.shtml";
                        Document news_document = get(url2, ProxyPool.proxy());

                        Elements ulList = news_document.select("#page").select("#Main").select("ul");
                        for (int n = 0; n < ulList.size(); n++) {
                            Elements liList = ulList.get(n).select("li");
                            for (int x = 0; x < liList.size(); x++) {
                                String news_url = liList.get(x).select("a").get(0).attr("href");
                                SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                CommonTaskManager.addTask("SinaNews", task, 1000);
                                System.out.println(news_url);
                            }
                        }

                        for (int j = 2; j <= 1000; j++) {
                            String new_url2 = "http://roll.finance.sina.com.cn/finance/gjcj/" + types[m] + "/index_" + j + ".shtml";
                            Document document = get(new_url2, ProxyPool.proxy());
                            Elements ulList2 = document.select("#page").select("#Main").select("ul");
                            for (int n = 0; n < ulList2.size(); n++) {
                                Elements liList = ulList2.get(n).select("li");
                                for (int x = 0; x < liList.size(); x++) {
                                    String news_url = liList.get(x).select("a").get(0).attr("href");
                                    SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                    CommonTaskManager.addTask("SinaNews", task, 1000);
                                    System.out.println(news_url);
                                }
                            }

                        }


                    } catch (Exception e) {
                        logger.error(e.getMessage());
                        e.printStackTrace();
                    }
                }*/

                    } else if ("娱乐".equals(name)) {
                        String url1 = "http://ent.sina.com.cn/rollnews.shtml#pageid=382&lid=48&k=&num=50&page=1";

                        try {
                            Document news_document = get(url1, ProxyPool.proxy());
                            Elements ulList = news_document.select("#vBody").select(".wrap").select("#pL_Main").select("#d_list").select("ul");
                            for (int m = 0; m < ulList.size(); m++) {
                                Elements liList = ulList.get(m).select("li");
                                for (int n = 0; n < liList.size(); n++) {
                                    String news_url = liList.get(n).select(".c_tit").select("a").get(0).attr("href");
                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                        CommonTaskManager.addTask("SinaNews", task, 1000);
                                        System.out.println(news_url);
                                    }
                                }
                            }

//                            int total_page = 0;
//                            Elements pageList = news_document.select("#vBody").select(".wrap").select("#pL_Main").select("#d_list").select(".pagebox").select(".pagebox_num");
//                            if (pageList.size() > 0 && pageList != null) {
//                                try {
//                                    total_page = Integer.parseInt(pageList.get(pageList.size() - 1).text());
//                                } catch (Exception e) {
//                                    e.printStackTrace();
//                                }
//                            }
//
//                            if (total_page >= 2) {
//                                for (int j = 2; j <= total_page; j++) {
//                                    try {
//                                        String new_url = "http://ent.sina.com.cn/rollnews.shtml#pageid=382&lid=48&k=&num=50&page=" + j;
//                                        Document news_document1 = get(new_url, ProxyPool.proxy());
//                                        Elements ulList1 = news_document1.select("#vBody").select(".wrap").select("#pL_Main").select("#d_list").select("ul");
//                                        for (int m = 0; m < ulList1.size(); m++) {
//                                            Elements liList = ulList1.get(m).select("li");
//                                            for (int n = 0; n < liList.size(); n++) {
//                                                String news_url = liList.get(n).select(".c_tit").select("a").get(0).attr("href");
//                                                if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                    SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                    CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                    System.out.println(news_url);
//                                                }
//                                            }
//                                        }
//                                    } catch (Exception e) {
//                                        logger.error(e.getMessage());
//                                        e.printStackTrace();
//                                        continue;
//                                    }
//                                }
//                            }

                        } catch (Exception e) {
                            logger.error(e.getMessage());
                            e.printStackTrace();
                        }

                    } else if ("科技".equals(name)) {
                        int[] lids = {21, 1195, 2128};
                        Date now = null;

                        for (int m = 0; m < lids.length; m++) {
                            now = new Date();
                            String url1 = "http://feed.mix.sina.com.cn/api/roll/get?pageid=1&lid=" + lids[m] + "&num=" + pageSize + "&versionNumber=1.2.8&page=1&encode=utf-8&callback=feedCardJsonpCallback&_=" + now.getTime();
                            boolean flag1 = true;
                            long future1 = System.currentTimeMillis() + Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));
                            long remianing1 = Long.valueOf(PropertiesUtil.getVal("SinaNews.GetForumUrlTime"));


                            while (flag1 && remianing1 >= 0) {
                                try {
                                    remianing1 = future1 - System.currentTimeMillis();
                                    Document document = get(url1, ProxyPool.proxy());
                                    //System.out.println(document.toString());
                                    Elements body = document.select("body");

                                    int begin = body.toString().indexOf("(");
                                    int end = body.toString().lastIndexOf(")");

                                    String new_body = body.toString().substring(begin+1,end);

                                    end = new_body.lastIndexOf(")");
                                    JSONObject object = JSON.parseObject(new_body.substring(0, end));
                                    JSONObject resultObject = object.getJSONObject("result");
                                    if(resultObject.get("total") == null){
                                        System.out.println(resultObject);
                                    }

                                    String total = resultObject.get("total").toString();
                                    JSONArray urlArray = resultObject.getJSONArray("data");

                                    for (int n = 0; n < urlArray.size(); n++) {
                                        JSONObject contentObject = urlArray.getJSONObject(n);
                                        String news_url = contentObject.get("url").toString();
                                        if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
                                            SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
                                            CommonTaskManager.addTask("SinaNews", task, 1000);
                                            System.out.println(news_url);
                                        }
                                    }

//                                    int total_count = 0;
//                                    int total_page = 0;
//                                    try {
//                                        total_count = Integer.parseInt(total);
//                                        total_page = (total_count - 1) / pageSize + 1;
//                                    } catch (Exception e) {
//                                        e.printStackTrace();
//                                    }
//
//                                    if (total_page >= 2) {
//                                        for (int j = 2; j <= total_page; j++) {
//                                            try {
//                                                now = new Date();
//                                                String new_url = "http://feed.mix.sina.com.cn/api/roll/get?pageid=1&lid=" + lids[m] + "&num=" + pageSize + "&versionNumber=1.2.8&page=" + j + "&encode=utf-8&callback=feedCardJsonpCallback&_=" + now.getTime();
//                                                Document news_document = get(new_url, ProxyPool.proxy());
//                                                Elements body1 = news_document.select("body");
//                                                int begin1 = body1.toString().indexOf("(");
//                                                int end1 = body1.toString().lastIndexOf(")");
//
//                                                String new_body1 = body1.toString().substring(begin1+1,end1);
//
//                                                end1 = new_body1.lastIndexOf(")");
//                                                JSONObject object1= JSON.parseObject(new_body1.substring(0, end1));
//                                                JSONObject resultObject1 = object1.getJSONObject("result");
//                                                JSONArray urlArray1 = resultObject1.getJSONArray("data");
//
//                                                for (int n = 0; n < urlArray1.size(); n++) {
//                                                    JSONObject contentObject = urlArray1.getJSONObject(n);
//                                                    String news_url = contentObject.get("url").toString();
//                                                    if(news_url.indexOf("slide.")<0&&news_url.indexOf("video.")<0){
//                                                        SinaNewsTask task = new SinaNewsTask(URLDecoder.decode(news_url, "UTF-8"), name);
//                                                        CommonTaskManager.addTask("SinaNews", task, 1000);
//                                                        System.out.println(news_url);
//                                                    }
//                                                }
//                                            } catch (Exception e) {
//                                                logger.error(e.getMessage());
//                                                e.printStackTrace();
//                                                continue;
//                                            }
//
//                                        }
//                                    }

                                    flag1 = false;
                                } catch (Exception e) {
                                    logger.error(e.getMessage());
                                    e.printStackTrace();
                                }
                            }

                        }
                    }
                }
            }catch(Exception e1){
                logger.error(e1.getMessage());
                e1.printStackTrace();
            }
        }
    }
}
